{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gen_train_data.ipynb  gen_train_data.py     test.ipynb\r\n"
     ]
    }
   ],
   "source": [
    "ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/lxp/tap4fun/data\n"
     ]
    }
   ],
   "source": [
    "cd ../data/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import gc\n",
    "import datetime\n",
    "\n",
    "#导入模型相关函数\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "#导入模型保存、模型评估等函数\n",
    "from sklearn.externals import joblib\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.metrics import mean_squared_error   #均方误差回归损失\n",
    "\n",
    "#划分测试和训练集\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "#获取rf_regressor 模型\n",
    "def train_model(train, feaList, labelName, model=None, saveToFile=None):\n",
    "    #定义模型\n",
    "    if model is None: #默认采用随机森林模型\n",
    "        model = RandomForestRegressor(n_estimators=10, criterion='mse', max_depth=None, min_samples_split=2, \n",
    "                                  min_samples_leaf=1, min_weight_fraction_leaf=0.0, max_features='auto', max_leaf_nodes=None,\n",
    "                                  min_impurity_decrease=0.0, min_impurity_split=None, \n",
    "                                  bootstrap=True, oob_score=False, n_jobs=4, random_state=None, verbose=0, warm_start=False)\n",
    "    \n",
    "    #训练集\n",
    "    X_train = train[feaList].values\n",
    "    y_train = train[[labelName]].values.ravel()\n",
    "    \n",
    "    # 拟合模型\n",
    "    model.fit(X_train, y_train)\n",
    "    \n",
    "    print ('train model ok')\n",
    "    \n",
    "    # 保存模型\n",
    "    if saveToFile is not None:\n",
    "        joblib.dump(model, saveToFile)\n",
    "        \n",
    "        print ('model saveToFile!')\n",
    "        \n",
    "    #删除数据，并且释放内存\n",
    "    del X_train, y_train\n",
    "    gc.collect()\n",
    "    \n",
    "    return model\n",
    "\n",
    "#测试对应模型，输出模型结果\n",
    "def test_model(test, model, feaList, labelName):\n",
    "    X_test = test[feaList].values\n",
    "    y_test = test[[labelName]].values.ravel()\n",
    "    \n",
    "    y_test_pred = model.predict(X_test)\n",
    "    \n",
    "    # 输出测试结果\n",
    "    mse = mean_squared_error(y_test, y_test_pred)\n",
    "    r2Score = r2_score(y_test, y_test_pred)\n",
    "    \n",
    "    print('MSE test: %.3f' % (mse))\n",
    "    print('R^2 test: %.3f' % (r2Score))\n",
    "    \n",
    "    #删除数据，并且释放内存\n",
    "    del X_test, y_test, y_test_pred\n",
    "    gc.collect()\n",
    "    \n",
    "    return mse, r2Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "runDate = datetime.datetime.now().strftime('%Y%m%d')\n",
    "\n",
    "model_name = 'gbrt'\n",
    "n_jobs_for_model = 4\n",
    "modelSavePath = model_name + 'model' + runDate + '.pickle'\n",
    "resultSavePath = model_name + 'result' + runDate + '.csv'\n",
    "nrows = None\n",
    "trainDataPath = 'train_data.csv'\n",
    "predictDataPath = 'predict_data.csv'\n",
    "\n",
    "feaList = [\n",
    "#  'user_id',\n",
    "#  'register_time',\n",
    " 'wood_add_value',\n",
    " 'wood_reduce_value',\n",
    " 'stone_add_value',\n",
    " 'stone_reduce_value',\n",
    " 'ivory_add_value',\n",
    " 'ivory_reduce_value',\n",
    " 'meat_add_value',\n",
    " 'meat_reduce_value',\n",
    " 'magic_add_value',\n",
    " 'magic_reduce_value',\n",
    " 'infantry_add_value',\n",
    " 'infantry_reduce_value',\n",
    " 'cavalry_add_value',\n",
    " 'cavalry_reduce_value',\n",
    " 'shaman_add_value',\n",
    " 'shaman_reduce_value',\n",
    " 'wound_infantry_add_value',\n",
    " 'wound_infantry_reduce_value',\n",
    " 'wound_cavalry_add_value',\n",
    " 'wound_cavalry_reduce_value',\n",
    " 'wound_shaman_add_value',\n",
    " 'wound_shaman_reduce_value',\n",
    " 'general_acceleration_add_value',\n",
    " 'general_acceleration_reduce_value',\n",
    " 'building_acceleration_add_value',\n",
    " 'building_acceleration_reduce_value',\n",
    " 'reaserch_acceleration_add_value',\n",
    " 'reaserch_acceleration_reduce_value',\n",
    " 'training_acceleration_add_value',\n",
    " 'training_acceleration_reduce_value',\n",
    " 'treatment_acceleraion_add_value',\n",
    " 'treatment_acceleration_reduce_value',\n",
    " 'bd_training_hut_level',\n",
    " 'bd_healing_lodge_level',\n",
    " 'bd_stronghold_level',\n",
    " 'bd_outpost_portal_level',\n",
    " 'bd_barrack_level',\n",
    " 'bd_healing_spring_level',\n",
    " 'bd_dolmen_level',\n",
    " 'bd_guest_cavern_level',\n",
    " 'bd_warehouse_level',\n",
    " 'bd_watchtower_level',\n",
    " 'bd_magic_coin_tree_level',\n",
    " 'bd_hall_of_war_level',\n",
    " 'bd_market_level',\n",
    " 'bd_hero_gacha_level',\n",
    " 'bd_hero_strengthen_level',\n",
    " 'bd_hero_pve_level',\n",
    " 'sr_scout_level',\n",
    " 'sr_training_speed_level',\n",
    " 'sr_infantry_tier_2_level',\n",
    " 'sr_cavalry_tier_2_level',\n",
    " 'sr_shaman_tier_2_level',\n",
    " 'sr_infantry_atk_level',\n",
    " 'sr_cavalry_atk_level',\n",
    " 'sr_shaman_atk_level',\n",
    " 'sr_infantry_tier_3_level',\n",
    " 'sr_cavalry_tier_3_level',\n",
    " 'sr_shaman_tier_3_level',\n",
    " 'sr_troop_defense_level',\n",
    " 'sr_infantry_def_level',\n",
    " 'sr_cavalry_def_level',\n",
    " 'sr_shaman_def_level',\n",
    " 'sr_infantry_hp_level',\n",
    " 'sr_cavalry_hp_level',\n",
    " 'sr_shaman_hp_level',\n",
    " 'sr_infantry_tier_4_level',\n",
    " 'sr_cavalry_tier_4_level',\n",
    " 'sr_shaman_tier_4_level',\n",
    " 'sr_troop_attack_level',\n",
    " 'sr_construction_speed_level',\n",
    " 'sr_hide_storage_level',\n",
    " 'sr_troop_consumption_level',\n",
    " 'sr_rss_a_prod_levell',\n",
    " 'sr_rss_b_prod_level',\n",
    " 'sr_rss_c_prod_level',\n",
    " 'sr_rss_d_prod_level',\n",
    " 'sr_rss_a_gather_level',\n",
    " 'sr_rss_b_gather_level',\n",
    " 'sr_rss_c_gather_level',\n",
    " 'sr_rss_d_gather_level',\n",
    " 'sr_troop_load_level',\n",
    " 'sr_rss_e_gather_level',\n",
    " 'sr_rss_e_prod_level',\n",
    " 'sr_outpost_durability_level',\n",
    " 'sr_outpost_tier_2_level',\n",
    " 'sr_healing_space_level',\n",
    " 'sr_gathering_hunter_buff_level',\n",
    " 'sr_healing_speed_level',\n",
    " 'sr_outpost_tier_3_level',\n",
    " 'sr_alliance_march_speed_level',\n",
    " 'sr_pvp_march_speed_level',\n",
    " 'sr_gathering_march_speed_level',\n",
    " 'sr_outpost_tier_4_level',\n",
    " 'sr_guest_troop_capacity_level',\n",
    " 'sr_march_size_level',\n",
    " 'sr_rss_help_bonus_level',\n",
    " 'pvp_battle_count',\n",
    " 'pvp_lanch_count',\n",
    " 'pvp_win_count',\n",
    " 'pve_battle_count',\n",
    " 'pve_lanch_count',\n",
    " 'pve_win_count',\n",
    " 'avg_online_minutes',\n",
    " 'pay_price',\n",
    " 'pay_count',\n",
    "#  'prediction_pay_price',\n",
    " 'year',\n",
    " 'month',\n",
    " 'day',\n",
    " 'hour',\n",
    " 'weekday'\n",
    "]\n",
    "\n",
    "labelName = 'prediction_pay_price'\n",
    "\n",
    "model_dict = {\n",
    "    'rf': RandomForestRegressor(n_estimators=100, criterion='mse', max_depth=None, min_samples_split=2, \n",
    "                              min_samples_leaf=1, min_weight_fraction_leaf=0.0, max_features='auto', max_leaf_nodes=None,\n",
    "                              min_impurity_decrease=0.0, min_impurity_split=None, \n",
    "                              bootstrap=True, oob_score=False, n_jobs=n_jobs_for_model, random_state=47, verbose=0, warm_start=False),\n",
    "\n",
    "    'gbrt':GradientBoostingRegressor(loss='ls', learning_rate=0.1, n_estimators=100, max_features='auto', max_depth=7, \n",
    "                                     subsample=1.0,criterion='friedman_mse', min_samples_split=2, min_samples_leaf=1,\n",
    "                                     min_weight_fraction_leaf=0.0, min_impurity_decrease=0.0,\n",
    "                                     min_impurity_split=None, init=None, random_state=47, \n",
    "                                     alpha=0.9, verbose=0, max_leaf_nodes=None, warm_start=False, presort='auto')\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#读取训练集\n",
    "df = pd.read_csv('tap_fun_train.csv', nrows=nrows)\n",
    "#处理注册时间，转化为datetime类型\n",
    "df.register_time = df.register_time.apply(pd.to_datetime, format='%Y-%m-%d %H:%M:%S')\n",
    "#获得注册信息（year, month, day, hour, weekday）\n",
    "df['year'] = df.register_time.apply(lambda x: x.year)\n",
    "df['month'] = df.register_time.apply(lambda x: x.month)\n",
    "df['day'] = df.register_time.apply(lambda x: x.day)\n",
    "df['hour'] = df.register_time.apply(lambda x: x.hour)\n",
    "df['weekday'] = df.register_time.apply(lambda x: x.weekday())\n",
    "\n",
    "oldFeaList = [\n",
    "#  'user_id',\n",
    "#  'register_time',\n",
    " 'wood_add_value',\n",
    " 'wood_reduce_value',\n",
    " 'stone_add_value',\n",
    " 'stone_reduce_value',\n",
    " 'ivory_add_value',\n",
    " 'ivory_reduce_value',\n",
    " 'meat_add_value',\n",
    " 'meat_reduce_value',\n",
    " 'magic_add_value',\n",
    " 'magic_reduce_value',\n",
    " 'infantry_add_value',\n",
    " 'infantry_reduce_value',\n",
    " 'cavalry_add_value',\n",
    " 'cavalry_reduce_value',\n",
    " 'shaman_add_value',\n",
    " 'shaman_reduce_value',\n",
    " 'wound_infantry_add_value',\n",
    " 'wound_infantry_reduce_value',\n",
    " 'wound_cavalry_add_value',\n",
    " 'wound_cavalry_reduce_value',\n",
    " 'wound_shaman_add_value',\n",
    " 'wound_shaman_reduce_value',\n",
    " 'general_acceleration_add_value',\n",
    " 'general_acceleration_reduce_value',\n",
    " 'building_acceleration_add_value',\n",
    " 'building_acceleration_reduce_value',\n",
    " 'reaserch_acceleration_add_value',\n",
    " 'reaserch_acceleration_reduce_value',\n",
    " 'training_acceleration_add_value',\n",
    " 'training_acceleration_reduce_value',\n",
    " 'treatment_acceleraion_add_value',\n",
    " 'treatment_acceleration_reduce_value',\n",
    " 'bd_training_hut_level',\n",
    " 'bd_healing_lodge_level',\n",
    " 'bd_stronghold_level',\n",
    " 'bd_outpost_portal_level',\n",
    " 'bd_barrack_level',\n",
    " 'bd_healing_spring_level',\n",
    " 'bd_dolmen_level',\n",
    " 'bd_guest_cavern_level',\n",
    " 'bd_warehouse_level',\n",
    " 'bd_watchtower_level',\n",
    " 'bd_magic_coin_tree_level',\n",
    " 'bd_hall_of_war_level',\n",
    " 'bd_market_level',\n",
    " 'bd_hero_gacha_level',\n",
    " 'bd_hero_strengthen_level',\n",
    " 'bd_hero_pve_level',\n",
    " 'sr_scout_level',\n",
    " 'sr_training_speed_level',\n",
    " 'sr_infantry_tier_2_level',\n",
    " 'sr_cavalry_tier_2_level',\n",
    " 'sr_shaman_tier_2_level',\n",
    " 'sr_infantry_atk_level',\n",
    " 'sr_cavalry_atk_level',\n",
    " 'sr_shaman_atk_level',\n",
    " 'sr_infantry_tier_3_level',\n",
    " 'sr_cavalry_tier_3_level',\n",
    " 'sr_shaman_tier_3_level',\n",
    " 'sr_troop_defense_level',\n",
    " 'sr_infantry_def_level',\n",
    " 'sr_cavalry_def_level',\n",
    " 'sr_shaman_def_level',\n",
    " 'sr_infantry_hp_level',\n",
    " 'sr_cavalry_hp_level',\n",
    " 'sr_shaman_hp_level',\n",
    " 'sr_infantry_tier_4_level',\n",
    " 'sr_cavalry_tier_4_level',\n",
    " 'sr_shaman_tier_4_level',\n",
    " 'sr_troop_attack_level',\n",
    " 'sr_construction_speed_level',\n",
    " 'sr_hide_storage_level',\n",
    " 'sr_troop_consumption_level',\n",
    " 'sr_rss_a_prod_levell',\n",
    " 'sr_rss_b_prod_level',\n",
    " 'sr_rss_c_prod_level',\n",
    " 'sr_rss_d_prod_level',\n",
    " 'sr_rss_a_gather_level',\n",
    " 'sr_rss_b_gather_level',\n",
    " 'sr_rss_c_gather_level',\n",
    " 'sr_rss_d_gather_level',\n",
    " 'sr_troop_load_level',\n",
    " 'sr_rss_e_gather_level',\n",
    " 'sr_rss_e_prod_level',\n",
    " 'sr_outpost_durability_level',\n",
    " 'sr_outpost_tier_2_level',\n",
    " 'sr_healing_space_level',\n",
    " 'sr_gathering_hunter_buff_level',\n",
    " 'sr_healing_speed_level',\n",
    " 'sr_outpost_tier_3_level',\n",
    " 'sr_alliance_march_speed_level',\n",
    " 'sr_pvp_march_speed_level',\n",
    " 'sr_gathering_march_speed_level',\n",
    " 'sr_outpost_tier_4_level',\n",
    " 'sr_guest_troop_capacity_level',\n",
    " 'sr_march_size_level',\n",
    " 'sr_rss_help_bonus_level',\n",
    " 'pvp_battle_count',\n",
    " 'pvp_lanch_count',\n",
    " 'pvp_win_count',\n",
    " 'pve_battle_count',\n",
    " 'pve_lanch_count',\n",
    " 'pve_win_count',\n",
    " 'avg_online_minutes',\n",
    " 'pay_price',\n",
    " 'pay_count',\n",
    "#  'prediction_pay_price',\n",
    "#  'year',\n",
    "#  'month',\n",
    "#  'day',\n",
    "#  'hour',\n",
    "#  'weekday'\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "newFeaList = [\n",
    "    #各种材料使用情况\n",
    "    'wood_usage',\n",
    "    'stone_usage',\n",
    "    'ivory_usage',\n",
    "    'meat_usage',\n",
    "    'magic_usage',\n",
    "    'infantry_usage',\n",
    "    'cavalry_usage',\n",
    "    'shaman_usage',\n",
    "    \n",
    "    #各种材料使用情况分布\n",
    "    'usage_max',\n",
    "    'usage_min',\n",
    "    'usage_mean',\n",
    "    'usage_std',\n",
    "    'usage_gap',\n",
    "    'usage_cv',\n",
    "\n",
    "    #恢复/消耗情况\n",
    "    'wound_infantry_renew',\n",
    "    'wound_cavalry_renew',\n",
    "    'wound_shaman_renew',\n",
    "    \n",
    "    #恢复/消耗分布\n",
    "    'renew_max',\n",
    "    'renew_std',\n",
    "    \n",
    "    #加速使用情况\n",
    "    'general_acceleration_acc_usage',\n",
    "    'building_acceleration_acc_usage',\n",
    "    'reaserch_acceleration_acc_usage',\n",
    "    'training_acceleration_acc_usage',\n",
    "    'treatment_acceleration_acc_usage',\n",
    "    \n",
    "    #加速使用情况分布\n",
    "    'acc_usage_max',\n",
    "    'acc_usage_min',\n",
    "    'acc_usage_mean',\n",
    "    'acc_usage_std',\n",
    "    'acc_usage_gap',\n",
    "    'acc_usage_cv',\n",
    "    \n",
    "    #建筑等级发展分布\n",
    "    'bd_level_max',\n",
    "    'bd_level_min',\n",
    "    'bd_level_mean',\n",
    "    'bd_level_std',\n",
    "    'bd_level_gap',\n",
    "    'bd_level_cv',\n",
    "    \n",
    "    #科研等级发展分布\n",
    "    'sr_level_max',\n",
    "    'sr_level_min',\n",
    "    'sr_level_mean',\n",
    "    'sr_level_std',\n",
    "    'sr_level_gap',\n",
    "    'sr_level_cv',\n",
    "    \n",
    "    #等级发展均衡性\n",
    "    'max_level_balance',\n",
    "    'mean_level_balance',\n",
    "    \n",
    "    #PVP/PVE主动发起比例\n",
    "    'pvp_battle_lanch_ratio',\n",
    "    'pve_battle_lanch_ratio',\n",
    "    \n",
    "    #PVP/PVE获胜比例\n",
    "    'pvp_battle_win_ratio',\n",
    "    'pve_battle_win_ratio',\n",
    "    \n",
    "    #主动发起的PVP/PVE获胜比例\n",
    "    'pvp_battle_lanch_win_ratio',\n",
    "    'pve_battle_lanch_win_ratio',\n",
    "    \n",
    "    #被动发起的PVP/PVE获胜比例\n",
    "    'pvp_battle_no_lanch_win_ratio',\n",
    "    'pve_battle_no_lanch_win_ratio'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#特征工程\n",
    "#各类物品的使用率，以usage结尾\n",
    "df['wood_usage'] = df['wood_reduce_value'] / (df['wood_add_value'] + 1)\n",
    "df['stone_usage'] = df['stone_reduce_value'] / (df['stone_add_value'] + 1)\n",
    "df['ivory_usage'] = df['ivory_reduce_value'] / (df['ivory_add_value'] + 1)\n",
    "df['meat_usage'] = df['meat_reduce_value'] / (df['meat_add_value'] + 1)\n",
    "df['magic_usage'] = df['magic_reduce_value'] / (df['magic_add_value'] + 1)\n",
    "df['infantry_usage'] = df['infantry_reduce_value'] / (df['infantry_add_value'] + 1)\n",
    "df['cavalry_usage'] = df['cavalry_reduce_value'] / (df['cavalry_add_value'] + 1)\n",
    "df['shaman_usage'] = df['shaman_reduce_value'] / (df['shaman_add_value'] + 1)\n",
    "\n",
    "#各类物品使用率均值、最大值、最小值\n",
    "df['usage_max'] = np.max(df[['wood_usage',\n",
    "    'stone_usage',\n",
    "    'ivory_usage',\n",
    "    'meat_usage',\n",
    "    'magic_usage',\n",
    "    'infantry_usage',\n",
    "    'cavalry_usage',\n",
    "    'shaman_usage']], axis=1)\n",
    "df['usage_mean'] = np.mean(df[['wood_usage',\n",
    "    'stone_usage',\n",
    "    'ivory_usage',\n",
    "    'meat_usage',\n",
    "    'magic_usage',\n",
    "    'infantry_usage',\n",
    "    'cavalry_usage',\n",
    "    'shaman_usage']], axis=1)\n",
    "df['usage_std'] = np.std(df[['wood_usage',\n",
    "    'stone_usage',\n",
    "    'ivory_usage',\n",
    "    'meat_usage',\n",
    "    'magic_usage',\n",
    "    'infantry_usage',\n",
    "    'cavalry_usage',\n",
    "    'shaman_usage']], axis=1)\n",
    "df['usage_min'] = np.min(df[['wood_usage',\n",
    "    'stone_usage',\n",
    "    'ivory_usage',\n",
    "    'meat_usage',\n",
    "    'magic_usage',\n",
    "    'infantry_usage',\n",
    "    'cavalry_usage',\n",
    "    'shaman_usage']], axis=1)\n",
    "df['usage_gap'] = df['usage_max'] - df['usage_min']\n",
    "df['usage_cv'] = df['usage_std'] / (df['usage_mean'] + 1e-6)\n",
    "\n",
    "#伤亡恢复情况，以renew结尾\n",
    "df['wound_infantry_renew'] = df['wound_infantry_reduce_value'] / (df['wound_infantry_add_value'] + 1)\n",
    "df['wound_cavalry_renew'] = df['wound_cavalry_reduce_value'] / (df['wound_cavalry_add_value'] + 1)\n",
    "df['wound_shaman_renew'] = df['wound_shaman_reduce_value'] / (df['wound_shaman_add_value'] + 1)\n",
    "\n",
    "df['renew_max'] = np.max(df[['wound_infantry_renew',\n",
    "    'wound_cavalry_renew',\n",
    "    'wound_shaman_renew']], axis=1)\n",
    "df['renew_std'] = np.std(df[['wound_infantry_renew',\n",
    "    'wound_cavalry_renew',\n",
    "    'wound_shaman_renew']], axis=1)\n",
    "\n",
    "#加速获取情况，以acc_usage结尾\n",
    "df['general_acceleration_acc_usage'] = df['general_acceleration_reduce_value'] / (df['general_acceleration_add_value'] + 1)\n",
    "df['building_acceleration_acc_usage'] = df['building_acceleration_reduce_value'] / (df['building_acceleration_add_value'] + 1)\n",
    "df['reaserch_acceleration_acc_usage'] = df['reaserch_acceleration_reduce_value'] / (df['reaserch_acceleration_add_value'] + 1)\n",
    "df['training_acceleration_acc_usage'] = df['training_acceleration_reduce_value'] / (df['training_acceleration_add_value'] + 1)\n",
    "df['treatment_acceleration_acc_usage'] = df['treatment_acceleration_reduce_value'] / (df['treatment_acceleraion_add_value'] + 1)\n",
    "\n",
    "df['acc_usage_max'] = np.max(df[['general_acceleration_acc_usage',\n",
    "    'building_acceleration_acc_usage',\n",
    "    'reaserch_acceleration_acc_usage',\n",
    "    'training_acceleration_acc_usage',\n",
    "    'treatment_acceleration_acc_usage']], axis=1)\n",
    "df['acc_usage_min'] = np.min(df[['general_acceleration_acc_usage',\n",
    "    'building_acceleration_acc_usage',\n",
    "    'reaserch_acceleration_acc_usage',\n",
    "    'training_acceleration_acc_usage',\n",
    "    'treatment_acceleration_acc_usage']], axis=1)\n",
    "df['acc_usage_mean'] = np.mean(df[['general_acceleration_acc_usage',\n",
    "    'building_acceleration_acc_usage',\n",
    "    'reaserch_acceleration_acc_usage',\n",
    "    'training_acceleration_acc_usage',\n",
    "    'treatment_acceleration_acc_usage']], axis=1)\n",
    "df['acc_usage_std'] = np.std(df[['general_acceleration_acc_usage',\n",
    "    'building_acceleration_acc_usage',\n",
    "    'reaserch_acceleration_acc_usage',\n",
    "    'training_acceleration_acc_usage',\n",
    "    'treatment_acceleration_acc_usage']], axis=1)\n",
    "df['acc_usage_gap'] = df['acc_usage_max'] - df['acc_usage_min']\n",
    "df['acc_usage_cv'] = df['acc_usage_std'] / (df['acc_usage_mean'] + 1e-6)\n",
    "\n",
    "#建筑等级分布情况\n",
    "bd_level_fea_list = ['bd_training_hut_level',\n",
    " 'bd_healing_lodge_level',\n",
    " 'bd_stronghold_level',\n",
    " 'bd_outpost_portal_level',\n",
    " 'bd_barrack_level',\n",
    " 'bd_healing_spring_level',\n",
    " 'bd_dolmen_level',\n",
    " 'bd_guest_cavern_level',\n",
    " 'bd_warehouse_level',\n",
    " 'bd_watchtower_level',\n",
    " 'bd_magic_coin_tree_level',\n",
    " 'bd_hall_of_war_level',\n",
    " 'bd_market_level',\n",
    " 'bd_hero_gacha_level',\n",
    " 'bd_hero_strengthen_level',\n",
    " 'bd_hero_pve_level'\n",
    "]\n",
    "df['bd_level_max'] = np.max(df[bd_level_fea_list], axis=1)\n",
    "df['bd_level_min'] = np.min(df[bd_level_fea_list], axis=1)\n",
    "df['bd_level_mean'] = np.mean(df[bd_level_fea_list], axis=1)\n",
    "df['bd_level_std'] = np.std(df[bd_level_fea_list], axis=1)\n",
    "df['bd_level_gap'] = df['bd_level_max'] - df['bd_level_min']\n",
    "df['bd_level_cv'] = df['bd_level_std'] / (df['bd_level_mean'] - 1e-2)\n",
    "\n",
    "#科研等级分布情况\n",
    "sr_level_fea_list = ['sr_scout_level',\n",
    " 'sr_training_speed_level',\n",
    " 'sr_infantry_tier_2_level',\n",
    " 'sr_cavalry_tier_2_level',\n",
    " 'sr_shaman_tier_2_level',\n",
    " 'sr_infantry_atk_level',\n",
    " 'sr_cavalry_atk_level',\n",
    " 'sr_shaman_atk_level',\n",
    " 'sr_infantry_tier_3_level',\n",
    " 'sr_cavalry_tier_3_level',\n",
    " 'sr_shaman_tier_3_level',\n",
    " 'sr_troop_defense_level',\n",
    " 'sr_infantry_def_level',\n",
    " 'sr_cavalry_def_level',\n",
    " 'sr_shaman_def_level',\n",
    " 'sr_infantry_hp_level',\n",
    " 'sr_cavalry_hp_level',\n",
    " 'sr_shaman_hp_level',\n",
    " 'sr_infantry_tier_4_level',\n",
    " 'sr_cavalry_tier_4_level',\n",
    " 'sr_shaman_tier_4_level',\n",
    " 'sr_troop_attack_level',\n",
    " 'sr_construction_speed_level',\n",
    " 'sr_hide_storage_level',\n",
    " 'sr_troop_consumption_level',\n",
    " 'sr_rss_a_prod_levell',\n",
    " 'sr_rss_b_prod_level',\n",
    " 'sr_rss_c_prod_level',\n",
    " 'sr_rss_d_prod_level',\n",
    " 'sr_rss_a_gather_level',\n",
    " 'sr_rss_b_gather_level',\n",
    " 'sr_rss_c_gather_level',\n",
    " 'sr_rss_d_gather_level',\n",
    " 'sr_troop_load_level',\n",
    " 'sr_rss_e_gather_level',\n",
    " 'sr_rss_e_prod_level',\n",
    " 'sr_outpost_durability_level',\n",
    " 'sr_outpost_tier_2_level',\n",
    " 'sr_healing_space_level',\n",
    " 'sr_gathering_hunter_buff_level',\n",
    " 'sr_healing_speed_level',\n",
    " 'sr_outpost_tier_3_level',\n",
    " 'sr_alliance_march_speed_level',\n",
    " 'sr_pvp_march_speed_level',\n",
    " 'sr_gathering_march_speed_level',\n",
    " 'sr_outpost_tier_4_level',\n",
    " 'sr_guest_troop_capacity_level',\n",
    " 'sr_march_size_level',\n",
    " 'sr_rss_help_bonus_level'\n",
    "]\n",
    "df['sr_level_max'] = np.max(df[sr_level_fea_list], axis=1)\n",
    "df['sr_level_min'] = np.min(df[sr_level_fea_list], axis=1)\n",
    "df['sr_level_mean'] = np.mean(df[sr_level_fea_list], axis=1)\n",
    "df['sr_level_std'] = np.std(df[sr_level_fea_list], axis=1)\n",
    "df['sr_level_gap'] = df['sr_level_max'] - df['sr_level_min']\n",
    "df['sr_level_cv'] = df['sr_level_std'] / (df['sr_level_mean'] - 1e-2)\n",
    "\n",
    "#等级发展均衡性\n",
    "df['max_level_balance'] = df['sr_level_max'] / df['bd_level_max']\n",
    "df['mean_level_balance'] = df['sr_level_mean'] / df['bd_level_mean']\n",
    "\n",
    "#PVP/PVE主动发起比例\n",
    "df['pvp_battle_lanch_ratio'] = df['pvp_lanch_count'] / (df['pvp_battle_count'] - 1e-2)\n",
    "df['pve_battle_lanch_ratio'] = df['pve_lanch_count'] / (df['pve_battle_count'] - 1e-2)\n",
    "\n",
    "#PVP/PVE获胜比例\n",
    "df['pvp_battle_win_ratio'] = df['pvp_win_count'] / (df['pvp_battle_count'] - 1e-2)\n",
    "df['pve_battle_win_ratio'] = df['pve_win_count'] / (df['pve_battle_count'] - 1e-2)\n",
    "\n",
    "#主动发起的PVP/PVE获胜比例\n",
    "df['pvp_battle_lanch_win_ratio'] = df['pvp_win_count'] / (df['pvp_lanch_count'] - 1e-2)\n",
    "df['pve_battle_lanch_win_ratio'] = df['pve_win_count'] / (df['pve_lanch_count'] - 1e-2)\n",
    "\n",
    "#被动发起的PVP/PVE获胜比例\n",
    "df['pvp_battle_no_lanch_win_ratio'] = df['pvp_win_count'] / (df['pvp_battle_count'] - df['pvp_lanch_count'] - 1e-2)\n",
    "df['pve_battle_no_lanch_win_ratio'] = df['pve_win_count'] / (df['pve_battle_count'] - df['pve_lanch_count'] - 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#获取各个特征的最大值、最小值、均值、标准差\n",
    "mean_dict = {}\n",
    "max_dict = {}\n",
    "min_dict = {}\n",
    "std_dict = {}\n",
    "\n",
    "for fea in (oldFeaList + newFeaList):\n",
    "    mean_dict[fea] = df[fea].mean()\n",
    "    max_dict[fea] = df[fea].max()\n",
    "    min_dict[fea] = df[fea].min()\n",
    "    std_dict[fea] = df[fea].std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#对训练数据进行归一化处理\n",
    "for fea in (oldFeaList + newFeaList):\n",
    "    df[fea] = (df[fea] - mean_dict[fea]) / (std_dict[fea] + 1e-6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#保存训练数据集\n",
    "df.to_csv(trainDataPath, index=None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train model ok\n",
      "model saveToFile!\n",
      "<bound method BaseEstimator.get_params of GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n",
      "             learning_rate=0.1, loss='ls', max_depth=7,\n",
      "             max_features='auto', max_leaf_nodes=None,\n",
      "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "             min_samples_leaf=1, min_samples_split=2,\n",
      "             min_weight_fraction_leaf=0.0, n_estimators=100,\n",
      "             presort='auto', random_state=47, subsample=1.0, verbose=0,\n",
      "             warm_start=False)>\n",
      "train: old= 2018-08-23 20:39:49.269275  new= 2018-08-23 22:39:30.667210 gap= 1:59:41.397935\n"
     ]
    }
   ],
   "source": [
    "#设置模型类型\n",
    "model = model_dict[model_name]\n",
    "#训练模型 \n",
    "df = df.fillna(0)\n",
    "oldtime=datetime.datetime.now()\n",
    "model = train_model(df, feaList, labelName, model=model, saveToFile=modelSavePath)\n",
    "newtime=datetime.datetime.now()\n",
    "print (model.get_params)\n",
    "print('train: old=', oldtime, ' new=', newtime, 'gap= %s'%(newtime - oldtime) )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#生成预测集样本\n",
    "pred = pd.read_csv('tap_fun_test.csv', nrows=nrows)\n",
    "#处理注册时间，转化为datetime类型\n",
    "pred.register_time = pred.register_time.apply(pd.to_datetime, format='%Y-%m-%d %H:%M:%S')\n",
    "#获得注册信息（year, month, day, hour, weekday）\n",
    "pred['year'] = pred.register_time.apply(lambda x: x.year)\n",
    "pred['month'] = pred.register_time.apply(lambda x: x.month)\n",
    "pred['day'] = pred.register_time.apply(lambda x: x.day)\n",
    "pred['hour'] = pred.register_time.apply(lambda x: x.hour)\n",
    "pred['weekday'] = pred.register_time.apply(lambda x: x.weekday())\n",
    "#对测试数据进行归一化处理\n",
    "for fea in (oldFeaList + newFeaList):\n",
    "    pred[fea] = (pred[fea] - mean_dict[fea]) / (std_dict[fea] + 1e-6)\n",
    "#保存测试数据集\n",
    "pred.to_csv(predictDataPath, index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#输出模型预测结果\n",
    "pred['prediction_pay_price'] = model.predict(pred[feaList].values)\n",
    "\n",
    "pred[['user_id', 'prediction_pay_price']].to_csv(resultSavePath, index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE test: 339.205\n",
      "R^2 test: 0.957\n",
      "test: old= 2018-08-23 22:41:31.394114  new= 2018-08-23 22:41:53.344492 gap= 0:00:21.950378\n"
     ]
    }
   ],
   "source": [
    "#测试模型\n",
    "oldtime=datetime.datetime.now()\n",
    "mse, r2Score = test_model(df, model, feaList, labelName)\n",
    "newtime=datetime.datetime.now()\n",
    "print('test: old=', oldtime, ' new=', newtime, 'gap= %s'%(newtime - oldtime) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['user_id', 'register_time', 'wood_add_value', 'wood_reduce_value',\n",
       "       'stone_add_value', 'stone_reduce_value', 'ivory_add_value',\n",
       "       'ivory_reduce_value', 'meat_add_value', 'meat_reduce_value',\n",
       "       ...\n",
       "       'max_level_balance', 'mean_level_balance', 'pvp_battle_lanch_ratio',\n",
       "       'pve_battle_lanch_ratio', 'pvp_battle_win_ratio',\n",
       "       'pve_battle_win_ratio', 'pvp_battle_lanch_win_ratio',\n",
       "       'pve_battle_lanch_win_ratio', 'pvp_battle_no_lanch_win_ratio',\n",
       "       'pve_battle_no_lanch_win_ratio'],\n",
       "      dtype='object', length=166)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
